﻿/*

Copyright (c) 2009, Stephen Remde
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of the <organization> nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY STEPHEN REMDE ''AS IS'' AND ANY
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL STEPHEN REMDE BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


*/

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml;
using System.Net;
using System.Text.RegularExpressions;
using System.IO;
using System.Threading;

namespace WikiMiner
{
    //class WikiMiner
    //{
    //    static List<string> questionterms = new List<string>();
    //    static float Mark(string answers, string a, string b)
    //    {
    //        float score = 0;

    //        string[] x = answers.Split(new string[] { "<br/>" }, StringSplitOptions.RemoveEmptyEntries);

    //        for (int i = 0; i < 5 && i < x.Length; i++)
    //        {
    //            string l = x[i].Trim().ToLower();

    //            if (l.StartsWith("http://en.wikipedia.org/wiki/"))
    //                l= l.Substring("http://en.wikipedia.org/wiki/".Length);



    //            float ra = scori[  a.ToLower() + "-"+l];
    //            float rb = scori[b.ToLower() + "-" + l];
    //            //float ra = Relatedness.Get(a, l);
    //            //float rb = Relatedness.Get(b, l);
                
    //            if (ra!=1 && rb !=1)
    //                score +=(float)Math.Sqrt( ra*rb);
                 
                
    //            //if (!questionterms.Contains(term))
    //                //questionterms.Add(term);

    //        }



    //        return score;
    //    }

    //    static Dictionary<string, float> scori = new Dictionary<string, float>();

    //    static void Main(string[] args)
    //    {
    //        string[] ans = File.ReadAllLines("c:\\pmi-marks.txt");
    //        foreach (var s in from a in ans select a.Split('\t'))
    //        {
    //            float f=0;
    //            float.TryParse(s[2], out f);
    //            scori[s[0] + "-" + s[1]] = f;
    //        }



    //        string[] lines = File.ReadAllText("c:\\res.html").Split(new string[]{"<tr>","</tr>"}, StringSplitOptions.RemoveEmptyEntries);
    //        List<string[]> cell = new List<string[]>();

    //        foreach (var l in lines)
    //        {
    //            var t = l.Split(new string[] { "<td class=hd><p style='height:16px;'>.</td><td  class='s2'>", "<td  class='s3'>" }, StringSplitOptions.None);
    //            if (t.Length>2)
    //                cell.Add(t.Skip(1).ToArray());
    //        }

    //        int[,,] counts1 = new int[10, 6, 3];
    //        int[,,] counts2 = new int[10, 6, 3];
    //        float[,] scores = new float[10,3];
    //        float[,] scores2 = new float[10, 3];

    //        string[] cats1 = new string[] { "Extremely Unconfident", "Very Unconfident", "Unconfident", "Confident", "Very Confident", "Extremely Confident" };
    //        string[] cats2 = new string[] { "Extremely Hard", "Very Hard", "Hard", "Easy", "Very Easy", "Extremely Easy" };

    //        string[] rooms = new string[] { "D1.2", "D4.1", "B2.2" };

    //        string[] cata1 = new string [] {"TERRORISM","MONET","YOUTUBE","ALAN TURING","CRYSTAL STRUCTURE","DEEP INELASTIC SCATTERING","ART","OPERATIONS RESEARCH","BANANA","MONTY PYTHON"};
    //        string[] cata2 = new string [] {"GLOBAL WARMING","CUMULUS","VIRUS","ELIZA","OPTIMISATION","BOHR MODEL","ALBACORE","COMBINATORIAL OPTIMISATION","COUNTER-INTELLIGENCE","UNIVERSE"};

               









    //        for (int i = 0; i < 10; i++)            
    //        {
    //            for (int j = 0; j < 59; j++)
    //            {
    //                counts1[i, Array.IndexOf<string>(cats1, cell[j][4 + i * 3]), Array.IndexOf<string>(rooms, cell[j][2].Substring(0,4))]++;
    //                counts2[i, Array.IndexOf<string>(cats2, cell[j][5 + i * 3]), Array.IndexOf<string>(rooms, cell[j][2].Substring(0, 4))]++;

    //                float score = Mark(cell[j][3 + i * 3], cata1[i], cata2[i]);
    //                scores[i, Array.IndexOf<string>(rooms, cell[j][2].Substring(0,4))] += score;
    //                scores2[i, Array.IndexOf<string>(rooms, cell[j][2].Substring(0, 4))] += score * score;
    //                cell[j][3 + i * 3] = score.ToString();
                    
    //            }

            

    //        }


            

    //        var url = "http://cwl-projects.cogsci.rpi.edu/cgi-bin/msr/msr.cgi";
    //        var wc = WebRequest.Create(url) as HttpWebRequest;
    //        wc.Method = "POST";
    //        wc.ContentType = "application/x-www-form-urlencoded";
            
    //        string postData = "msr=NSS-Gwikipedia&username=stephen&terms=[" + string.Join(",", questionterms.ToArray()) + "]";
    //        //postData = postData.Replace("&#39;", "'");
    //        //postData = postData.Replace("&amp;", "&");
    //        byte[] byteArray = Encoding.UTF8.GetBytes(postData);
    //        Stream dataStream = wc.GetRequestStream();
    //        dataStream.Write(byteArray, 0, byteArray.Length);
    //        dataStream.Close();
    //        WebResponse response = wc.GetResponse();
    //        Console.WriteLine(((HttpWebResponse)response).StatusDescription);
    //        dataStream = response.GetResponseStream();
    //        StreamReader reader = new StreamReader(dataStream);
    //        string responseFromServer = reader.ReadToEnd();
    //        Console.WriteLine(responseFromServer);
    //        reader.Close();
    //        dataStream.Close();
    //        response.Close();

            

    //        List<string> output = new List<string>();

    //        for (int i = 0; i < 10; i++)
    //        {
    //            float clink =0;
    //            float search=0;

    //            for (int j = 0; j < 6; j++)
    //            {
    //                    if  (i<5) 
    //                    {
    //                        clink += counts1[i,j,0];
    //                        search +=  (counts1[i,j,1] + counts1[i,j,2]);
    //                    }    
    //                    else
    //                    {
    //                        search += counts1[i,j,0];
    //                        clink +=  (counts1[i,j,1] + counts1[i,j,2]);
    //                    }
                
    //            }

    //            output.Add("");
    //            output.Add("");
    //            output.Add("Confidence Q" + (i+1) + ",CLink,Search");

    //            for (int j = 0; j < 6; j++)
    //            {
    //                if  (i<5) 
    //                {
    //                    output.Add(cats1[j] + "," + (counts1[i, j, 0]) / clink + "," + (counts1[i, j, 1] + counts1[i, j, 2]) / search);
    //                }
    //                else
    //                {
    //                    output.Add(cats1[j] + "," + (counts1[i,j,1] + counts1[i,j,2])/clink + "," + counts1[i,j,0]/search);
    //                }
    //            }

    //            output.Add("");
    //            output.Add("Ease Q" + (i + 1) + ",CLink,Search");

    //            for (int j = 0; j < 6; j++)
    //            {
    //                if (i < 5)
    //                {
    //                    output.Add(cats2[j] + "," + (counts2[i, j, 0]) / clink + "," + (counts2[i, j, 1] + counts2[i, j, 2]) / search);
    //                }
    //                else
    //                {
    //                    output.Add(cats2[j] + "," + (counts2[i, j, 1] + counts2[i, j, 2]) / clink + "," + counts2[i, j, 0] / search);
    //                }
    //            }

    //            output.Add("");
    //            output.Add("Score Q" + (i + 1) + ",CLink,Search");

    //            for (int j = 0; j < 10; j++)
    //            {
    //                float ccount = 0;
    //                float scount = 0;
    //                int ctot = 0;
    //                int stot = 0;

    //                for (int c = 0; c < 59; c++)
    //                {
    //                     float score = float.Parse(cell[c][3 + i * 3]);
    //                    int room = Array.IndexOf<string>(rooms, cell[c][2].Substring(0, 4));

    //                    if (score >= j/2.0 && score < j/2.0 + 0.5)
    //                    {
    //                        if ((room == 0) ^ (i > 4))
    //                        {
    //                            ccount++;
    //                        }
    //                        else
    //                        {
    //                            scount++;
    //                        }
    //                    }

    //                    if ((room == 0) ^ (i > 4))
    //                    {
    //                        ctot++;
    //                    }
    //                    else
    //                    {
    //                        stot++;
    //                    }
                        
    //                }
                    
    //                output.Add((j/2.0)  + " to " +(j/2.0+0.5) + " ," + (ccount / ctot) + "," + (scount/stot));
    //            }

    //            float set1_2 = scores2[i, 0];
    //            float set2_2 = scores2[i, 1] + scores2[i, 2];
    //            float set1_t = scores[i, 0] ;
    //            float set2_t = scores[i, 1] + scores[i, 2];

    //            if (i < 5)
    //            {
    //                output.Add("x," + (set1_t / clink) + "," + (set2_t / search));
    //                output.Add("s," + (
    //                    Math.Sqrt(set1_2 / clink - Math.Pow(set1_t / clink, 2))) + "," +
    //                    Math.Sqrt(set2_2 / search - Math.Pow(set2_t / search, 2))
    //                    );
    //            }
    //            else
    //            {

    //                output.Add("x," + (set2_t / clink) + "," + (set1_t / search));
    //                output.Add("s," + (
    //                    Math.Sqrt(set2_2 / clink- Math.Pow(set2_t / clink, 2))) + "," +
    //                    Math.Sqrt(set1_2 / search - Math.Pow(set1_t / search, 2))
    //                    );
    //            }
    //            output.Add("n," + clink + "," + search);
                

    //        }

    //        var output2 = from x in cell select string.Join(",", (from y in x select "\"" + y + "\"").ToArray());

            
    //        File.WriteAllLines("c:\\confidence+ease.csv", output.ToArray());
    //        File.WriteAllLines("c:\\copy2.csv", output2.ToArray());
    //            /*
    //        for (int i=1; i<=10; i++)
    //        {
    //            string[] lines = File.ReadAllLines("c:\\" + i + ".txt");

    //            List<string> output = new List<string>();

    //            string ta = lines[0];
    //            string tb = lines[1];
                
    //            var unique = (from b in lines.Skip(2) 
    //                          select b.ToLower().Replace("_"," ").Trim()
    //                          ).Distinct();

    //            output.Add("," + ta + "," + tb + ", multiple");

    //            var data = from term in unique
    //                       select new
    //                       {
    //                           term = term,
    //                           definition = Search.GetDescription(term),
    //                           ra = Relatedness.Get(ta, term),
    //                           rb = Relatedness.Get(tb, term)
    //                       };

    //            foreach (var entry in data.OrderBy(e => -e.ra * e.rb))
    //                output.Add(
    //                    "\"" + entry.term + "\"," + 
    //                    entry.ra.ToString("0.000") + "," + 
    //                    entry.rb.ToString("0.000") + "," + 
    //                    (entry.rb*entry.ra).ToString("0.000") +",\""+
    //                    entry.definition + "\""
    //                    );                

    //            File.WriteAllLines("c:\\" + i + ".csv", output.ToArray());
    //        }

    //        Console.ReadLine();
    //        */
    //    }

    //    private static Article[] DoSearch(string a, string b)
    //    {
    //        StreamWriter outp = new StreamWriter(a + "_TO_" + b);

    //        // search for the articles
    //        var sres = Search.Get(a).OrderBy(item => -item.commoness);
    //        var dres = Search.Get(b).OrderBy(item => -item.commoness);

    //        var source = sres.First();
    //        var dest = dres.First();
    //        outp.WriteLine("Starting Search from '{0}' to '{1}'\r\n\r\n{2}\r\n\r\n{3}\r\n", source.title, dest.title, ToText(source.description), ToText(dest.description));
    //        Console.WriteLine("Starting Search from '{0}' to '{1}'", source.title, dest.title);
    //        outp.WriteLine();

    //        List<Candidate> open = new List<Candidate>();
    //        List<Article> closed = new List<Article>();
                        
    //        open.Add(new Candidate() { article = source, destination = dest, parent = null });

    //        int its = 0;
    //        while (open.Count > 0)
    //        {
    //            its++; 
    //            Candidate at = open.OrderBy(item => item.f).First();

    //            closed.Add(at.article);
    //            open.Remove(at);


    //            outp.WriteLine("----------------------------------");
    //            outp.WriteLine("#" + its.ToString("000")  +" at (G=" + at.G.ToString("0.000") + ", H=" + at.H.ToString("0.000") + ") " + at.article.title);
    //            Console.WriteLine("#" + its.ToString("000") + " at (G=" + at.G.ToString("0.000") + ", H=" + at.H.ToString("0.000") + ") " + at.article.title);
    //            {
    //                Candidate at2 = at;

    //                List<Article> path = new List<Article>();

    //                while (at2 != null)
    //                {
    //                    path.Add(at2.article);
    //                    at2 = at2.parent;
    //                }

    //                path.Reverse();

    //                for (int i = 0; i < path.Count - 1; i++)
    //                {
    //                    outp.Write(path[i].title + " -- " + Relatedness.Get(path[i + 1].title, path[i].title).ToString("0.0%") + " --> ");
    //                }
    //                outp.WriteLine(path[path.Count - 1].title);
    //            }
    //            /*
    //            if (at.H == 1000 )
    //            {

    //                Console.WriteLine("No path found (H=1000).");

    //                outp.WriteLine("No path found (H=1000).");

    //                break;
    //            }
    //            else */if (its >= 500)
    //            {
    //                Console.WriteLine("No path found (its=" + its + ").");

    //                outp.WriteLine("No path found (its=" + its + ").");

    //                break;
    //            }
    //            else if (at.article.title != dest.title)
    //            {
    //                foreach (var link in Links.GetOutLinks(at.article.id))
    //                {
    //                    if (!closed.Exists(item => item.id == link.id))
    //                    {
    //                        Candidate existing = open.FirstOrDefault(item => item.article.id == link.id);

    //                        if (existing != null)
    //                        {
    //                            existing.UpdateParentIfBetter(at);
    //                        }
    //                        else
    //                        {
    //                            open.Add(new Candidate()
    //                            {
    //                                article = link,
    //                                parent = at
    //                            });
    //                        }
    //                    }
    //                }
    //            }
    //            else
    //            {
    //                List<Article> path = new List<Article>();

    //                while (at != null)
    //                {
    //                    path.Add(at.article);
    //                    at = at.parent;
    //                }

    //                path.Reverse();

    //                outp.WriteLine();
    //                outp.WriteLine("Found Path from '{0}' to '{1}' in {2} iterations", source.title, dest.title, its);
    //                Console.WriteLine("Found Path from '{0}' to '{1}' in {2} iterations", source.title, dest.title, its);

    //                for (int i = 0; i < path.Count - 1; i++)
    //                {
    //                    outp.Write(path[i].title + " -- " + Relatedness.Get(path[i + 1].title, path[i].title).ToString("0.0%") + " --> ");
    //                    Console.Write(path[i].title + " -- " + Relatedness.Get(path[i + 1].title, path[i].title).ToString("0.0%") + " --> ");
    //                }
    //                outp.WriteLine(dest.title);
    //                Console.WriteLine(dest.title);


    //                outp.Close();

    //                return path.ToArray();
    //            }
    //        }


    //        outp.WriteLine("No path found.");

    //        outp.Close();

    //        Relatedness.Save();

    //        return null;

    //    }

    //    private static object ToText(string p)
    //    {
    //        Regex.Replace(p, @"<[^>]*>", "");
    //        //Regex.Replace(p, @"[\s\r\n]+", " ");
    //        return p;
    //    }
    //    class Candidate
    //    {
    //        public Article article;
    //        public Article destination;
 
    //        public float G;
    //        public float H;
    //        public float f;

    //        public void Calc()
    //        {
    //            H = (1 - Relatedness.Get(article.title, destination.title));
    //            if (H == 1)
    //                H = 1000;


    //            f = G + H;             
    //        }


    //        private Candidate _parent = null;

    //        public Candidate parent
    //        {
    //            set
    //            {
    //                _parent = value;
    //                if (value == null)
    //                {
    //                    G = 0;
    //                }
    //                else
    //                {
    //                    G = _parent.G + 1 - Relatedness.Get(article.title, _parent.article.title);
    //                    destination = parent.destination;
    //                }
    //                Calc();
    //            }
    //            get
    //            {
    //                return _parent;
    //            }
    //        }

    //        internal void UpdateParentIfBetter(Candidate newparent)
    //        {
    //            if (_parent != null)
    //            {
    //                if (_parent.G + 1 - Relatedness.Get(article.title, newparent.article.title) < G)
    //                {
    //                    parent = newparent;
    //                }
    //            }
    //        }
    //    }
    //}
}
